package com.xiaotu.spider.project.pageprocessor.list;

import com.xiaotu.common.util.*;
import com.xiaotu.spider.AUrlPageProcessor;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;

/**
 * 爱奇艺列列表数据爬取：网大名称，上映年份，类型，点播量，评分,影片简介，时长
 */
@Component("IQYListPageProcessor")
public class IQYListPageProcessor extends AUrlPageProcessor
{
    //分类型的列表地址
    private static final String BASE_URL = "http://list.iqiyi.com/www/1/-%s-----27401-----%s--4-1-1-iqiyi--.html";
    //不分类型的列表地址
    //    private static final String BASE_URL = "http://list.iqiyi.com/www/1/------27401-----%s--4-1-1-iqiyi--.html";

    private static final String PLAY_URL = "http://mixer.video.iqiyi.com/jp/mixin/videos/%s?callback=window.Q" +
            ".__callbacks__.cbqt4y3n&status=1";
    private static final String[] YEARS = DateUtils.getRecentYears(3);

    private static final String[] TYPES = new String[]{"8" + SepratorUtil.SEP_UNDERLINE + "喜剧", "13" + SepratorUtil
            .SEP_UNDERLINE + "悲剧", "6" + SepratorUtil.SEP_UNDERLINE + "爱情", "11" + SepratorUtil.SEP_UNDERLINE + "动作",
            "131" + SepratorUtil.SEP_UNDERLINE + "枪战", "291" + SepratorUtil.SEP_UNDERLINE + "犯罪", "128" + SepratorUtil
            .SEP_UNDERLINE + "惊悚", "10" + SepratorUtil.SEP_UNDERLINE + "恐怖", "289" + SepratorUtil.SEP_UNDERLINE + "悬疑",
            "12" + SepratorUtil.SEP_UNDERLINE + "动画", "27356" + SepratorUtil.SEP_UNDERLINE + "家庭", "1284" +
            SepratorUtil.SEP_UNDERLINE + "奇幻", "129" + SepratorUtil.SEP_UNDERLINE + "魔幻", "9" + SepratorUtil
            .SEP_UNDERLINE + "科幻", "7" + SepratorUtil.SEP_UNDERLINE + "战争", "130" + SepratorUtil.SEP_UNDERLINE + "青春"};

    private static final String[] DEFAULT_SCORE = new String[]{StringUtils.EMPTY, StringUtils.EMPTY};


    @Override
    public void process(Page page)
    {
        if (page.getUrl().toString().contains("list.iqiyi.com"))
        {
            this.listProcess(page);
            this.addNextPage(page);
        }
        else
            detailProcess(page);
    }

    private void detailProcess(Page page)
    {
        String[] dataArr = this.getBaseInfo(page);
        if (dataArr == null)
            return;
        String line = StringUtils.join(dataArr, SepratorUtil.SEP_CSV_FIELDS);
        System.out.println(line);
        FileUtil.writeFile("D://iqy.csv", line + SepratorUtil.LINE_SEPRATOR);
    }

    private void listProcess(Page page)
    {
        List<String> list = page.getHtml().xpath("//div[@class='wrapper-piclist']//li").all();
        list.stream().forEach(divStr ->
        {
            Html html = new Html(divStr);
            page.getRequest().putExtra("score", HtmlUtil.delHTMLTag(html.xpath("//span[@class='score']").get()));
            page.getRequest().putExtra("videoTime", HtmlUtil.delHTMLTag(html.xpath("//span[@class='icon-vInfo']/text" +
                    "()").get()));
            PageUtils.addTargetRequest(page, html.xpath("//div[@class='site-piclist_pic']//a/@href").get(), page
                    .getRequest().getExtras(), true);
        });
    }

    private String[] getBaseInfo(Page page)
    {
        Html html = page.getHtml();
        String title = html.xpath("//span[@id='widget-videotitle']/text()").get();
        if (StringUtils.isEmpty(title))
        {
            FileUtil.writeFile("d:\\url.csv", page.getUrl().toString() + SepratorUtil.LINE_SEPRATOR);
            return null;
        }
        String type = StringUtils.join(html.xpath("//span[@id='datainfo-taglist']/a/text()").all().iterator(),
                SepratorUtil.SEP_COMMA_EN);
        String year = page.getRequest().getExtra("year").toString();
        String score = StringUtil.nullToString(page.getRequest().getExtra("score"));
        String desc = html.xpath("//span[@id='data-videoInfoDes']/text()").get();
        String[] playArr = this.getScore(this.getDataId(html));
        String videoTime = StringUtil.nullToString(page.getRequest().getExtra("videoTime"));
        return new String[]{page.getUrl().toString(), page.getRequest().getExtra("type").toString(), title, year, type,
                playArr[0], score, desc, playArr[1], videoTime};
    }

    private String[] getScore(String dataId)
    {
        if (StringUtils.isEmpty(dataId))
            return DEFAULT_SCORE;
        String url = String.format(PLAY_URL, dataId);
        String data = HttpUtils.httpGetJSONP(url);
        if (StringUtils.isEmpty(data))
            return new String[]{StringUtils.EMPTY, url};
        String play = RegexUtils.regexSearch("\"playCount\":" + RegexUtils.REGEX_INTEGER, data);
        if (StringUtils.isNotEmpty(play))
            return new String[]{RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, play), url};
        return new String[]{StringUtils.EMPTY, url};
    }

    private String getDataId(Html html)
    {
        String dataId = html.xpath("//div[@'data-player-albumid']/@data-player-albumid").get();
        if (StringUtils.isEmpty(dataId))
        {
            dataId = RegexUtils.regexSearch("albumId:(" + RegexUtils.REGEX_INTEGER + ")", html.get());
            dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, dataId);
        }
        return dataId;
    }

    private void addNextPage(Page page)
    {
        String href = page.getHtml().xpath("//div[@class='mod-page']/a[@data-key='down']/@href").get();
        if (StringUtils.isNotEmpty(href))
            PageUtils.addTargetRequest(page, href, page.getRequest().getExtras());

    }

    @Override
    protected List<JSONObject> getStartRequestUrlsAndParams()
    {
        List<JSONObject> list = new ArrayList<>();

        //分类型的列表地址
        Stream.of(YEARS).forEach(year -> Stream.of(TYPES).forEach(type ->
        {
            String[] typeArr = type.split(SepratorUtil.SEP_UNDERLINE);
            JSONObject target = new JSONObject();
            target.put("year", year);
            target.put("type", typeArr[1]);
            target.put(URL_KEY, String.format(BASE_URL, typeArr[0], year));
            list.add(target);
        }));

        //不分类型的列表地址
        //        Stream.of(YEARS).forEach(year ->
        //        {
        //            JSONObject target = new JSONObject();
        //            target.put("year", year);
        //            target.put("type", StringUtils.EMPTY);
        //            target.put(URL_KEY, String.format(BASE_URL, year));
        //            list.add(target);
        //        });
        return list;
    }

    @Override
    protected String[] getStartRequestUrls()
    {
        return null;
    }

    @Override
    protected String getSpiderType()
    {
        return "IQY_LIST";
    }

    @Override
    public String getPageDataType(Page page)
    {
        return Constants.PageDataType.SEARCH_LIST;
    }
}
